from pymongo import MongoClient
from selenium import webdriver
from bs4 import BeautifulSoup
from pprint import pprint
import requests
import datetime
import json
import time
import re

from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

driver = webdriver.Chrome(executable_path = '/Users/xuchaosheng/Workspace/KNX/libs/chromedriver')

# def is_element_visible(element):
#     try:
#         the_element = EC.visibility_of_element_located(element)
#         assert the_element(driver)
#         flag = True
#     except:
#         flag = False
#
#     return flag


client = MongoClient('47.104.130.19', 27017)
db = client['knx_posts_db']
offical_posts_coll = db['offical_posts_coll']


class POST():
    def __init__(self):
        self.scrapy()

    def scrapy(self):
        with open('demo.txt', 'r') as f:
            content = f.read()
            urls = content.split('https')

            for url in urls[1:]:
                try:
                    url = 'https' + url
                    driver.get(url)
                    time.sleep(2)
                    soup = BeautifulSoup(driver.page_source, 'lxml')
                    name = soup.find(class_ = 'jobtitleInJobDetails').text
                    desc0 = 'Job Description:' + soup.find(text = re.compile('Job Description')).parent.find_next_sibling().get_text()
                    desc1 = 'Minimum requirements:' + soup.find(text = re.compile('Minimum requirements')).parent.find_next_sibling().get_text()

                    location = soup.select('.section2RightfieldsInJobDetails')[3].text

                    item = {
                        "url": url,
                        'edu': '',
                        'exp': [],
                        'name': name,
                        'date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
                        'lang': '',
                        'place': '',
                        'major': '',
                        'count': '',
                        'salary': [],
                        'toSchool': True,
                        'welfare': [],
                        'funType': '',
                        'company': '诺华集团',
                        'location': location,
                        'industry': '医药',
                        'keywords': [],
                        'platform': 'offical',
                        'searchKeyword': '',
                        'description': desc0 + '\n' + desc1,
                        'subIndustry': '',
                        'stime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                    }

                    pprint(item)

                    result = offical_posts_coll.replace_one({'name': name, 'location': location}, item, True)

                    # if result.inserted_id:
                    #     print(result.inserted_id)
                except:
                    pass


POST()
